/*
 * This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
 *
 * Copyright (c) 2001 Brian Pitcher
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

// $Header: /cvsroot/weblech/weblech/src/weblech/spider/HTMLParser.java,v 1.3 2002/06/09 11:02:36 weblech Exp $

package wendyeq.iweb.blog.util.spider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpVersion;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.conn.ClientConnectionManager;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpParams;
import org.apache.http.params.HttpProtocolParams;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import wendyeq.iweb.blog.pojo.Article;
import wendyeq.iweb.blog.util.DateUtil;

public class HTMLParser
{
	private static final Logger _logClass = Logger.getLogger(HTMLParser.class);
	private static HTMLParser hp=new HTMLParser();
	public static HTMLParser getInstance(){
		if(hp==null)
			hp=new HTMLParser();
		return hp;
	}
	private HTMLParser(){
		
	}
	/****
	 * 处理url及相关内容
	 * 如果为index则获取page信息，如果是page则不获取翻页信息
	 * */
	public List<URL> parseLinksInDocument(URL sourceURL, String textContent)
	{
		//		if(pageFilter(sourceURL.toString())){
		return parseAsHTML(sourceURL, getIndexPage(sourceURL.toString(),textContent));
		//		}else
		//			return parseAsHTML(sourceURL, setPage(sourceURL.toString(),textContent));
	}
	
	/****
	 * 
	 * 从page中生成obj
	 * */
	public Article parseArticleInDocument(URL sourceURL, String textContent)
	{
		Article a=new Article();
		
		//		if(pageFilter(sourceURL.toString())){
		return a;
	}
	/**
	 * 处理索引页信息
	 * http://208.94.244.98/bt/thread.php?fid=16&page=1
	 * http://208.94.244.98/bt/thread.php?fid=16&page=2
	 * 
	 * 由于索引首页含有置顶信息，所以需要做定制处理
	 * **/
	private String getIndexPage(String url,String str){
		String index="bt/thread.php?fid=16";
		StringBuffer sb=new StringBuffer();
		String time=DateUtil.getDate();
//		time =DateUtil.timeplus(-2);
		if(url.contains(index)){
			Document doc = Jsoup.parse(str); 
			Elements div =doc.getElementsByClass("tr3");
			if(div.size()>0){
				sb.append("<HTML>");
				for (Iterator<Element> iterator = div.iterator(); iterator.hasNext();) {
					Element e = (Element) iterator.next();
					Elements el=e.select("div.f10");
					if(el.size()>0){
						//						System.out.println("el.size="+el.size());
						_logClass.info("|"+el.first().text()+"|"	);
						if(time.equals(el.first().text().trim())){
							sb.append(e.html());
						}

					}
				}
				sb.append("</HTML>");
			}
			//根据时间过滤置顶与过期数据
			return sb.toString();

		}
		return str;
	}
	
	/***
	 * 过滤指定url 而非如爬虫一样对所有url进行解析
	 * */
	private List<URL> parseAsHTML(URL sourceURL, String textContent){
		_logClass.info("sourceURL===>"+sourceURL.toString());
		_logClass.info(textContent);
		ArrayList<URL> newURLs = new ArrayList<URL>();
		
		Document doc = Jsoup.parse(textContent); 
		Elements div=doc.getElementsByTag("h3");
		for (Iterator<Element> iterator = div.iterator(); iterator.hasNext();) {
			Element e = (Element) iterator.next();
			Elements links = e.getElementsByTag("a");
			String url=null;
			for (Element link : links) {
			   url = link.attr("href");
			}
			_logClass.info("url===>"+url);
			try {
				newURLs.add(new URL("http://208.94.244.98/bt/"+url));
			} catch (MalformedURLException e1) {
				// TODO Auto-generated catch block
				e1.printStackTrace();
			}
		}
		return newURLs;
	}
	/***
	 * 过滤page的url
	 * */
	private List<URL> parseAsHTMLs(URL sourceURL, String textContent)
	{
		_logClass.info("parseAsHTML()");
		ArrayList<URL> newURLs = new ArrayList<URL>();
		HashSet<URL> newURLSet = new HashSet<URL>();


		extractAttributesFromTags("a", "href", sourceURL, newURLs, newURLSet, textContent);
		extractAttributesFromTags("A", "HREF", sourceURL, newURLs, newURLSet, textContent);

		//		extractAttributesFromTags("IMG", "SRC", sourceURL, newURLs, newURLSet, textContent);
		//		extractAttributesFromTags("img", "src", sourceURL, newURLs, newURLSet, textContent);
		//		extractAttributesFromTags("BODY", "BACKGROUND", sourceURL, newURLs, newURLSet, textContent);
		//		extractAttributesFromTags("body", "background", sourceURL, newURLs, newURLSet, textContent);
		//		extractAttributesFromTags("FRAME", "SRC", sourceURL, newURLs, newURLSet, textContent);
		//		extractAttributesFromTags("frame", "src", sourceURL, newURLs, newURLSet, textContent);

		if(newURLs.size() == 0)
		{
			_logClass.info("Got 0 new URLs from HTML parse, check HTML\n" + textContent);
		}
		_logClass.info("Returning " + newURLs.size() + " urls extracted from page");
		if(newURLs.size()>1){
			for (Iterator<URL> iterator = newURLSet.iterator(); iterator.hasNext();) {
				_logClass.info("url====> " + (URL) iterator.next());

			}

		}
		return newURLs;
	}
//	private List<URL> parseAsHTML(URL sourceURL, String textContent)
//	{
//		_logClass.info("parseAsHTML()");
//		ArrayList<URL> newURLs = new ArrayList<URL>();
//		HashSet<URL> newURLSet = new HashSet<URL>();
//
//
//		extractAttributesFromTags("a", "href", sourceURL, newURLs, newURLSet, textContent);
//		extractAttributesFromTags("A", "HREF", sourceURL, newURLs, newURLSet, textContent);
//
//		//		extractAttributesFromTags("IMG", "SRC", sourceURL, newURLs, newURLSet, textContent);
//		//		extractAttributesFromTags("img", "src", sourceURL, newURLs, newURLSet, textContent);
//		//		extractAttributesFromTags("BODY", "BACKGROUND", sourceURL, newURLs, newURLSet, textContent);
//		//		extractAttributesFromTags("body", "background", sourceURL, newURLs, newURLSet, textContent);
//		//		extractAttributesFromTags("FRAME", "SRC", sourceURL, newURLs, newURLSet, textContent);
//		//		extractAttributesFromTags("frame", "src", sourceURL, newURLs, newURLSet, textContent);
//
//		if(newURLs.size() == 0)
//		{
//			_logClass.info("Got 0 new URLs from HTML parse, check HTML\n" + textContent);
//		}
//		_logClass.info("Returning " + newURLs.size() + " urls extracted from page");
//		if(newURLs.size()>10){
//			for (Iterator<URL> iterator = newURLSet.iterator(); iterator.hasNext();) {
//				_logClass.info("url====> " + (URL) iterator.next());
//
//			}
//
//		}
//		return newURLs;
//	}

	private void extractAttributesFromTags(String tag, String attr, URL sourceURL, List<URL> newURLs, Set<URL> newURLSet, String input)
	{
		_logClass.info("extractAttributesFromTags(" + tag + ", " + attr + ", ...)");

		int startPos = 0;
		String startTag = "<" + tag + " ";
		String attrStr = attr + "=\"";
		while(true)
		{
			int tagPos = input.indexOf(startTag, startPos);
			if(tagPos < 0)
			{
				return;
			}
			int attrPos = input.indexOf(attrStr, tagPos + 1);
			if(attrPos < 0)
			{
				startPos = tagPos + 1;
				continue;
			}
			int nextClosePos = input.indexOf(">", tagPos + 1);
			if(attrPos < nextClosePos)
			{
				// Ooh, found one
				int closeQuotePos = input.indexOf("\"", attrPos + attrStr.length() + 1);
				if(closeQuotePos > 0)
				{
					String urlStr = input.substring(attrPos + attrStr.length(), closeQuotePos);
					if(urlStr.indexOf('#') != -1)
					{
						urlStr = urlStr.substring(0, urlStr.indexOf('#'));
					}
					//_logClass.info("Found possible URL string: " + URL);

					try
					{

						URL u = new URL(sourceURL, urlStr);
						if(newURLSet.contains(u))
						{
							//_logClass.info("Already found URL on page: " + u);
						}
						else
						{
							newURLs.add(u);
							newURLSet.add(u);
							//_logClass.info("Found new URL on page: " + u);
						}
					}
					catch(MalformedURLException murle)
					{
					}
				}
				startPos = tagPos + 1;
				continue;
			}
			else
			{
				startPos = tagPos + 1;
				continue;
			}
		}
	}

	//	private void logMailURL(String url)
	//	{
	//		_logClass.info("logMailURL()");
	//
	//		try
	//		{
	//			FileWriter appendedFile = new FileWriter(config.getMailtoLogFile().toString(), true);
	//			PrintWriter pW = new PrintWriter(appendedFile);
	//			pW.println(url);
	//			pW.flush();
	//			pW.close();
	//		}
	//		catch(IOException ioe)
	//		{
	//			_logClass.warn("Caught IO exception writing mailto URL:" + ioe.getMessage(), ioe);
	//		}
	//	}

	public String getPageCode(String purl){
		_logClass.info("访问链接"+purl);
		URL url;
		try {
			url = new URL(purl);
			InputStreamReader isr=new InputStreamReader(url.openStream());
			BufferedReader br = new BufferedReader(isr);
			String s = "";
			StringBuffer sb = new StringBuffer("");
			while ((s = br.readLine()) != null) {
				sb.append(s+"\r\n");
			}
			br.close();
			isr.close();
			//			System.out.println("得到网页源代码为"+sb.toString());
			return removeJS(sb.toString());
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return "";

	}
	/**
	 * 过滤获取的html中的js与空格
	 * */
	public static String removeJS(String str){
		String regEx_script = "<[\\s]*?script[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?script[\\s]*?>"; //定义script的正则表达式{或<script[^>]*?>[\\s\\S]*?<\\/script> }
		//		String js="<(\\s*|/)script(.*[^>]|)>";
		Pattern p = Pattern.compile(regEx_script);    
		Matcher m = p.matcher(str);    
		//		System.out.println("****************************************");
		str= m.replaceAll(""); // 过滤script标签    
		str=str.replace("&nbsp;","");
		return str;
	}
	public static String tagfilter(String str,String tag){
		String regEx_tag = "<[\\s]*?"+tag+"[^>]*?>[\\s\\S]*?<[\\s]*?\\/[\\s]*?"+tag+"[\\s]*?>"; //定义script的正则表达式{或<script[^>]*?>[\\s\\S]*?<\\/script> }
		Pattern p = Pattern.compile(regEx_tag);    
		Matcher m = p.matcher(str);    
		//		System.out.println("****************************************");
		str= m.replaceAll(""); // 过滤script标签    
		return str;
	}
	/***
	 * 使用HttpClient伪装成浏览器方式访问，
	 * */
	public static String getPageByHttpClient(String url,String codec){
		HttpParams params = new BasicHttpParams();
		//		HttpConnectionManagerParams.setMaxTotalConnections(params, 1000);// 超时设置
		ConnManagerParams.setMaxTotalConnections(params, 20);
		HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
		// Create and initialize scheme registry
		SchemeRegistry schemeRegistry = new SchemeRegistry();
		schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
		ClientConnectionManager cm = new ThreadSafeClientConnManager(params,schemeRegistry);
		DefaultHttpClient httpclient = new DefaultHttpClient(cm, params);
		httpclient.getParams().setParameter(ClientPNames.COOKIE_POLICY,
				CookiePolicy.BROWSER_COMPATIBILITY); // 设置cookie的兼容性
		HttpGet httpget = new HttpGet(url);
		StringBuilder sb=new StringBuilder();
		HttpResponse response ;
		HttpEntity entity;
		httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.2)");
		try {
			response = httpclient.execute(httpget);
			entity = response.getEntity();
			if (entity != null) {
				InputStream instream = entity.getContent();
				int l;
				byte[] tmp = new byte[2048];
				while ((l = instream.read(tmp)) != -1) {
					sb.append(new String(tmp,codec));
				}
			}
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return sb.toString();
	}
	public static void main(String ad[]){
		String page=getPageByHttpClient("http://208.94.244.98/bt/thread.php?fid=16&page=1","GBK");
		System.out.println("*********************");
		Document doc = Jsoup.parse(page); 
		Elements div = doc.getElementsByClass("tr3");
		//		Elements div2 = doc.select("tr.tr3 t_one");
		//		Elements div3 = doc.getElementsByClass("tr3 t_one");
		System.out.println(div.size());
		//		System.out.println(div2.size());
		//		System.out.println(div3.size());
		StringBuffer sb=new StringBuffer();
		String time=DateUtil.getDate();
		time=DateUtil.timeplus(-2);
		int i=0;
		if(div.size()>0){
			sb.append("<HTML>");
			for (Iterator<Element> iterator = div.iterator(); iterator.hasNext();) {
				Element e = (Element) iterator.next();
				Elements el=e.select("div.f10");
				if(el.size()>0){
					System.out.println("el.size="+el.size());
					//					System.out.println(	);
					if(time.equals(el.first().text())){
						i++;
						sb.append(e.html());
					}

				}

			}
			sb.append("</HTML>");

			System.out.println(sb.toString());
			System.out.println("迭代"+i+"次");
		}

	}
}
